""""数据探索"""

import jieba
import pandas as pd

from src.Commons.base_config_loader import BaseConfigLoader

def rf_load_data(config : BaseConfigLoader,data_path) -> pd.DataFrame:
    """加载训练数据"""
    data_dir = config.get(data_path)
    df = pd.read_csv(data_dir, sep='\t', names=['text', 'label'])

    # print(df.head())
    # print(len(df.text))
    # label_counter = Counter(df["label"])
    # label_len = len(df["label"])
    # print(label_counter)
    # for label, count in label_counter.items():
    #     print(f"标签：{label},占比：{count/label_len:.2f}")

    # 分析文本长度
    # df["text_len"] = df["text"].str.len()
    # len_mean = round(df["text_len"].mean(), 2)
    # len_std = round(df["text_len"].std(), 2)
    # len_min = df["text_len"].min()
    # len_max = df["text_len"].max()
    #
    # print(f"文本平均长度：{len_mean}， 加权平均：{len_std}, 最小长度：{len_min}, 最大长度：{len_max}")
    #
    # df["text_len"].hist()
    # plt.show()
    # df["label"].hist()
    # plt.show()
    return df

def cat_sentence(context: str) -> str:
    return " ".join(jieba.lcut(context))[:30]


def rf_process_data(df:pd.DataFrame,config, target_path: str):
    """将给定的数据，按照text的文本，固定长度分词后，保存到指定文件路径下"""
    df["word"] = df["text"].apply(cat_sentence)
    data_dir = config.get(target_path)

    df.to_csv(data_dir, index=False)
    del df

def df_init_data(config: BaseConfigLoader):
    """初始化数据"""
    # 原始文件路径
    train_path = "system.resources.src_data.train_path"
    test_path = "system.resources.src_data.test_path"
    dev_path = "system.resources.src_data.dev_path"

    # 数据处理后的文件，即是按照指定长度切词后的文件路径
    train_process_path = "system.resources.random_forest.process_data.train_path"
    test_process_path = "system.resources.random_forest.process_data.test_path"
    dev_process_path = "system.resources.random_forest.process_data.dev_path"

    # 加载原始数据
    train_data: pd.DataFrame = rf_load_data(config, train_path)
    rf_process_data(train_data,config, train_process_path)

    test_data: pd.DataFrame = rf_load_data(config, test_path)
    rf_process_data(test_data, config, test_process_path)

    dev_data: pd.DataFrame = rf_load_data(config, dev_path)
    rf_process_data(dev_data, config, dev_process_path)

if __name__ == '__main__':
    config_loader = BaseConfigLoader()

    df_init_data(config_loader)



